# base encoder settings that can be extended and overriden
# we leave out the attention part for other config to override

_target_: xformers.factory.block_factory.xFormerDecoderConfig
reversible: False  # Optionally make these layers reversible to save memory
num_layers: 3  # Optional this means that this config will repeat N times
block_type: decoder
dim_model: ${emb}
residual_norm_style: pre  # Optional pre/post
position_encoding_config:
  name: vocab  # whatever position encodinhg makes sense
  seq_len: ${seq}
  vocab_size: ${vocab}
  dropout: 0
multi_head_config_masked:
  num_heads: 4
  residual_dropout: 0
  attention: ???
multi_head_config_cross:
  num_heads: 4
  residual_dropout: 0
  attention: ???
feedforward_config:
  name: MLP
  dropout: 0
  activation: relu
  hidden_layer_multiplier: 4
